<!DOCTYPE html>
<html lang="zh-CN">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>数据验证测试工具</title>
    <style>
        body { font-family: Arial, sans-serif; margin: 20px; }
        .section { margin: 20px 0; padding: 15px; border: 1px solid #ccc; border-radius: 5px; }
        .result { background: #f0f0f0; padding: 10px; margin: 10px 0; border-radius: 3px; }
        .error { background: #ffebee; color: #c62828; }
        .success { background: #e8f5e8; color: #2e7d32; }
        table { border-collapse: collapse; width: 100%; }
        th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
        th { background-color: #f2f2f2; }
    </style>
</head>
<body>
    <h1>考试题目数据验证测试</h1>
    
    <div class="section">
        <h2>测试数据加载</h2>
        <button onclick="loadTestData()">加载51CTO.csv并验证</button>
        <div id="testResults"></div>
    </div>

    <div class="section">
        <h2>原始数据预览</h2>
        <div id="rawDataPreview"></div>
    </div>

    <div class="section">
        <h2>处理后数据预览</h2>
        <div id="processedDataPreview"></div>
    </div>

    <script>
        async function loadTestData() {
            const results = document.getElementById('testResults');
            results.innerHTML = '<p>正在加载数据...</p>';

            try {
                // 读取CSV文件
                const response = await fetch('爬虫资料/51CTO.csv');
                const csvText = await response.text();
                
                // 原始数据解析
                const originalData = parseCSV(csvText);
                
                // 显示原始数据统计
                const originalStats = analyzeOriginalData(originalData);
                
                // 使用工具的预处理逻辑
                const processedData = preprocessData(originalData);
                
                // 显示处理结果
                displayResults(originalStats, processedData, originalData);
                
            } catch (error) {
                results.innerHTML = `<div class="result error">错误: ${error.message}</div>`;
            }
        }

        function parseCSV(text) {
            const lines = text.trim().split('\n');
            if (lines.length === 0) return [];

            const headers = lines[0].split(',').map(h => h.trim().replace(/^"|"$/g, ''));
            const data = [];

            for (let i = 1; i < lines.length; i++) {
                const line = lines[i];
                if (!line.trim()) continue;

                // 处理引号内的逗号
                const values = [];
                let current = '';
                let inQuotes = false;
                
                for (let char of line) {
                    if (char === '"') {
                        inQuotes = !inQuotes;
                    } else if (char === ',' && !inQuotes) {
                        values.push(current.trim().replace(/^"|"$/g, ''));
                        current = '';
                    } else {
                        current += char;
                    }
                }
                values.push(current.trim().replace(/^"|"$/g, ''));

                const row = {};
                headers.forEach((header, index) => {
                    row[header] = values[index] || '';
                });
                data.push(row);
            }

            return data;
        }

        function analyzeOriginalData(data) {
            let totalRecords = data.length;
            let emptyQuestions = 0;
            let multiLineRecords = 0;
            let currentRecord = null;

            data.forEach((row, index) => {
                const hasQuestion = row['题目描述'] && row['题目描述'].trim();
                
                if (hasQuestion) {
                    if (currentRecord) multiLineRecords++;
                    currentRecord = row;
                } else if (currentRecord) {
                    // 这是多行记录的一部分
                } else {
                    emptyQuestions++;
                }
            });

            return {
                totalLines: totalRecords,
                expectedCompleteRecords: totalRecords - emptyQuestions,
                emptyQuestionLines: emptyQuestions,
                potentialMultiLineRecords: multiLineRecords
            };
        }

        function preprocessData(data) {
            // 复制工具的预处理逻辑
            const redundantColumns = ['web-scraper-order', 'web-scraper-start-url'];
            data.forEach(row => {
                redundantColumns.forEach(col => delete row[col]);
            });

            const mergedData = [];
            let currentRecord = null;

            data.forEach(row => {
                const hasQuestion = row['题目描述'] && row['题目描述'].trim();
                
                if (hasQuestion) {
                    if (currentRecord) {
                        mergedData.push(currentRecord);
                    }
                    currentRecord = { ...row };
                } else if (currentRecord) {
                    // 合并到当前记录
                    Object.keys(row).forEach(key => {
                        if (row[key] && row[key].trim()) {
                            if (currentRecord[key] && currentRecord[key].trim()) {
                                currentRecord[key] += '\n' + row[key];
                            } else {
                                currentRecord[key] = row[key];
                            }
                        }
                    });
                }
            });

            if (currentRecord) {
                mergedData.push(currentRecord);
            }

            return mergedData;
        }

        function displayResults(originalStats, processedData, originalData) {
            const results = document.getElementById('testResults');
            
            const html = `
                <div class="result success">
                    <h3>数据验证结果</h3>
                    <p><strong>原始文件总行数:</strong> ${originalStats.totalLines + 1} (包含表头)</p>
                    <p><strong>原始数据记录数:</strong> ${originalStats.totalLines}</p>
                    <p><strong>预期完整记录数:</strong> ${originalStats.expectedCompleteRecords}</p>
                    <p><strong>空题目行数:</strong> ${originalStats.emptyQuestionLines}</p>
                    <p><strong>潜在多行记录:</strong> ${originalStats.potentialMultiLineRecords}</p>
                    <p><strong>实际处理后记录数:</strong> ${processedData.length}</p>
                    <p><strong>记录合并效果:</strong> ${originalStats.totalLines - processedData.length} 条记录被合并</p>
                </div>
            `;
            
            results.innerHTML = html;

            // 显示原始数据预览
            const rawPreview = document.getElementById('rawDataPreview');
            rawPreview.innerHTML = `
                <h4>原始数据前10行:</h4>
                <table>
                    <tr><th>行号</th><th>题目描述</th><th>回答情况</th><th>状态</th></tr>
                    ${originalData.slice(0, 10).map((row, index) => `
                        <tr>
                            <td>${index + 1}</td>
                            <td>${(row['题目描述'] || '').substring(0, 50)}${(row['题目描述'] || '').length > 50 ? '...' : ''}</td>
                            <td>${row['回答情况'] || '无'}</td>
                            <td>${row['题目描述'] && row['题目描述'].trim() ? '完整题目' : '空题目/续行'}</td>
                        </tr>
                    `).join('')}
                </table>
            `;

            // 显示处理后数据预览
            const processedPreview = document.getElementById('processedDataPreview');
            processedPreview.innerHTML = `
                <h4>处理后数据前10条记录:</h4>
                <table>
                    <tr><th>序号</th><th>题目描述</th><th>回答情况</th><th>题目长度</th></tr>
                    ${processedData.slice(0, 10).map((row, index) => `
                        <tr>
                            <td>${index + 1}</td>
                            <td>${(row['题目描述'] || '').substring(0, 50)}${(row['题目描述'] || '').length > 50 ? '...' : ''}</td>
                            <td>${row['回答情况'] || '无'}</td>
                            <td>${(row['题目描述'] || '').length} 字符</td>
                        </tr>
                    `).join('')}
                </table>
            `;
        }

        // 页面加载完成后自动测试
        window.addEventListener('load', loadTestData);
    </script>
</body>
</html>